#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @FileName  :run_type_list.py
# @Time      :2023/10/12 
# @Author    :CL
# @email     :1037654919@qq.com
from multiprocessing import Pool

from bs4 import BeautifulSoup

from douban import douban
from utils import mongo_manager, MongoDBUtil


def get_type_code():
    print('get_type_code begin')
    res = '''<div>
      <h2>分类排行榜 · · · · · ·<img style=" position: absolute;" src="https://img1.doubanio.com/f/shire/e49eca1517424a941871a2667a8957fd6c72d632/pics/new_menu.gif"></h2>
      <div class="types">
          <span><a href="/typerank?type_name=剧情&amp;type=11&amp;interval_id=100:90&amp;action=">剧情</a></span>
          <span><a href="/typerank?type_name=喜剧&amp;type=24&amp;interval_id=100:90&amp;action=">喜剧</a></span>
          <span><a href="/typerank?type_name=动作&amp;type=5&amp;interval_id=100:90&amp;action=">动作</a></span>
          <span><a href="/typerank?type_name=爱情&amp;type=13&amp;interval_id=100:90&amp;action=">爱情</a></span>
          <span><a href="/typerank?type_name=科幻&amp;type=17&amp;interval_id=100:90&amp;action=">科幻</a></span>
          <span><a href="/typerank?type_name=动画&amp;type=25&amp;interval_id=100:90&amp;action=">动画</a></span>
          <span><a href="/typerank?type_name=悬疑&amp;type=10&amp;interval_id=100:90&amp;action=">悬疑</a></span>
          <span><a href="/typerank?type_name=惊悚&amp;type=19&amp;interval_id=100:90&amp;action=">惊悚</a></span>
          <span><a href="/typerank?type_name=恐怖&amp;type=20&amp;interval_id=100:90&amp;action=">恐怖</a></span>
          <span><a href="/typerank?type_name=纪录片&amp;type=1&amp;interval_id=100:90&amp;action=">纪录片</a></span>
          <span><a href="/typerank?type_name=短片&amp;type=23&amp;interval_id=100:90&amp;action=">短片</a></span>
          <span><a href="/typerank?type_name=情色&amp;type=6&amp;interval_id=100:90&amp;action=">情色</a></span>
          <span><a href="/typerank?type_name=音乐&amp;type=14&amp;interval_id=100:90&amp;action=">音乐</a></span>
          <span><a href="/typerank?type_name=歌舞&amp;type=7&amp;interval_id=100:90&amp;action=">歌舞</a></span>
          <span><a href="/typerank?type_name=家庭&amp;type=28&amp;interval_id=100:90&amp;action=">家庭</a></span>
          <span><a href="/typerank?type_name=儿童&amp;type=8&amp;interval_id=100:90&amp;action=">儿童</a></span>
          <span><a href="/typerank?type_name=传记&amp;type=2&amp;interval_id=100:90&amp;action=">传记</a></span>
          <span><a href="/typerank?type_name=历史&amp;type=4&amp;interval_id=100:90&amp;action=">历史</a></span>
          <span><a href="/typerank?type_name=战争&amp;type=22&amp;interval_id=100:90&amp;action=">战争</a></span>
          <span><a href="/typerank?type_name=犯罪&amp;type=3&amp;interval_id=100:90&amp;action=">犯罪</a></span>
          <span><a href="/typerank?type_name=西部&amp;type=27&amp;interval_id=100:90&amp;action=">西部</a></span>
          <span><a href="/typerank?type_name=奇幻&amp;type=16&amp;interval_id=100:90&amp;action=">奇幻</a></span>
          <span><a href="/typerank?type_name=冒险&amp;type=15&amp;interval_id=100:90&amp;action=">冒险</a></span>
          <span><a href="/typerank?type_name=灾难&amp;type=12&amp;interval_id=100:90&amp;action=">灾难</a></span>
          <span><a href="/typerank?type_name=武侠&amp;type=29&amp;interval_id=100:90&amp;action=">武侠</a></span>
          <span><a href="/typerank?type_name=古装&amp;type=30&amp;interval_id=100:90&amp;action=">古装</a></span>
          <span><a href="/typerank?type_name=运动&amp;type=18&amp;interval_id=100:90&amp;action=">运动</a></span>
          <span><a href="/typerank?type_name=黑色电影&amp;type=31&amp;interval_id=100:90&amp;action=">黑色电影</a></span>
      </div>
  </div>'''
    soups = BeautifulSoup(res, 'lxml')
    datas = soups.find('div', class_='types').find_all('span')
    results = []
    for data in datas:
        name = data.text
        href = data.find('a')['href']
        type_code = str(href).split('type=')[1].split('&')[0]
        results.append({'name': name, 'href': href, 'type_code': type_code})
    # pd.DataFrame(results).to_excel('douban_type_code.xlsx')
    return results


type_code = get_type_code()


def down_type_list(seed):
    print('down_type_list(seed)', seed)
    # seed = {'code':code['type_code'],'interval_id':interval_id}
    code = seed['code']
    interval_id = seed['interval_id']
    params = {
        "type": code,
        "interval_id": interval_id
    }
    totals = douban.get_total(params=params)
    for j in range(0, totals, 20):
        params2 = {
            "type": code,
            "interval_id": interval_id,
            "action": "",
            "start": j,
            "limit": "20"
        }
        type_list = douban.get_type_list(params=params2)
        for data in type_list:
            data['_id'] = code + '_' + data['id']
            data['code'] = code
            try:
                # print(data)
                MongoDBUtil.insert_one('douban_type_list', data)
            except Exception as e:
                print(e)
    # seed['_id'] = seed['code'] +'_'+ seed['interval_id']
    seed['status'] = 'success'
    try:
        # print(data)
        MongoDBUtil.insert_one('douban_type_list_seed', seed)
    except Exception as e:
        MongoDBUtil.update_one('douban_type_list_seed', {'_id': seed["_id"]}, {'$set': seed})


def run_type_list():
    # 获取豆瓣 type_code
    douban_type_list_seed = mongo_manager('douban_type_list_seed', db='douban')
    # type_code = get_type_code()
    # seeds = []
    # for code in type_code:
    #     print('code', code)
    #     for i in range(100, 0, -10):
    #         interval_id = str(i) + ':' + str(i - 10)
    #         seeds.append({'name': code['name'],'code': code['type_code'], 'interval_id': interval_id})
    # for seed in seeds:
    #     seed['_id'] = seed['code'] + '_' + seed['interval_id']
    #     # seed['status'] = 'success'
    #     try:
    #         # print(data)
    #         douban_type_list_seed.insertOne(seed)
    #     except Exception as e:
    #         douban_type_list_seed.updateOne({'_id': seed["_id"]}, seed)
    seeds = douban_type_list_seed.findAll({'status': None})
    lists = []
    for seed in seeds:
        lists.append(seed)
    # run
    print('_' * 20, len(lists))
    pool = Pool(processes=min(30, len(lists) + 1))
    pool.map(down_type_list, lists)
    pool.close()  # 关闭进程池，不再接受新的进程
    pool.join()


if __name__ == "__main__":
    run_type_list()
